/*
 * TOP SECRET Copyright 2006-2015 Transsion.com All right reserved. This software is the confidential and proprietary
 * information of Transsion.com ("Confidential Information"). You shall not disclose such Confidential Information and
 * shall use it only in accordance with the terms of the license agreement you entered into with Transsion.com.
 */
package com.yunji.framework_template.common.http;

/**
 * ClassName:IconFinder <br/>
 * 注：获取Icon的公共Api:https://blog.csdn.net/swanabin/article/details/46660433 <br>
 * Date: 2018年11月14日 上午10:04:35 <br/>
 * 
 * @author fenglibin1982@163.com
 * @Blog http://blog.csdn.net/fenglibing
 * @version
 * @see
 */
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;

import com.yunji.framework_template.common.util.StringUtil;
import com.yunji.framework_template.common.web.HttpResult;

public class WebIconFinder {

    private static final Logger    logger           = Logger.getLogger(WebIconFinder.class);
    private static final Pattern[] ICON_PATTERNS    = new Pattern[] { Pattern.compile("rel=[\"']icon[\"'][^\r\n>]+?((?<=href=[\"']).+?(?=[\"']))"),
                                                                      Pattern.compile("rel=[\"']shortcut icon[\"'][^\r\n>]+?((?<=href=[\"']).+?(?=[\"']))"),
                                                                      Pattern.compile("((?<=link href=[\"']).+?(?=[\"']))[^\r\n<]+?rel=[\"']shortcut icon[\"']"),
                                                                      Pattern.compile("((?<=link href=[\"']).+?(?=[\"']))[^\r\n<]+?rel=[\"']icon[\"']"),
                                                                      Pattern.compile("rel=[\"']icon prefetch[\"'][^\r\n>]+?((?<=href=[\"']).+?(?=[\"']))") };

    private static final Pattern   HEAD_END_PATTERN = Pattern.compile("</head>");

    public static void main(String[] args) throws IOException {
        long startTime = System.currentTimeMillis();

        Set<String> urlSet = getUrls("http://www.hao123.com/");
        for (String urlString : urlSet) {
            System.out.println(urlString);
            System.out.println(getIconUrlString(urlString));
        }

        System.out.println(urlSet.size());
        System.out.println("耗时：" + (System.currentTimeMillis() - startTime) + " ms");
    }

    // 获取Icon地址
    public static String getIconUrlString(String urlString) {
        try {
            urlString = getRealUrl(urlString);
            String icoUrl = null;
            int tryTimes = 0;
            while (true) {
                tryTimes++;
                logger.info("tryTimes:" + tryTimes);
                URL url = new URL(urlString);
                // String iconUrl = url.getProtocol() + "://" + url.getHost() + "/favicon.ico";// 保证从域名根路径搜索
                // if (hasRootIcon(iconUrl)) return iconUrl;
                icoUrl = getIconUrlByRegex(urlString);
                if (icoUrl == null || icoUrl.length() == 0) {
                    if (tryTimes == 1) {
                        if (!url.getHost().startsWith("www")) {
                            urlString = url.getProtocol() + "://" + "www." + url.getHost();
                        }
                    }
                    if (tryTimes == 2) {
                        if ("https" != url.getProtocol()) {
                            urlString = "https://" + url.getHost();
                        }
                    }

                } else {
                    return icoUrl;
                }
                if (tryTimes >= 3) {
                    break;
                }
            }
            return icoUrl;
        } catch (Exception e) {
            logger.error("Get icon by url:" + urlString + " exceptioned.", e);
            return null;
        }
    }

    // 判断在根目录下是否有Icon
    private static boolean hasRootIcon2(String urlString) {
        HttpURLConnection connection = null;

        try {
            connection = getConnection(urlString);
            connection.connect();
            return HttpURLConnection.HTTP_OK == connection.getResponseCode() && connection.getContentLength() > 0;
        } catch (Exception e) {
            logger.error("check root icon exception, url:" + urlString, e);
            return false;
        } finally {
            if (connection != null) connection.disconnect();
        }
    }

    // 判断在根目录下是否有Icon
    private static boolean hasRootIcon(String urlString) {
        try {
            HttpResult result = HttpClientUtil.checkUrl(urlString, null);
            if (result.getCode() == 200) {
                return true;
            }
            return false;
        } catch (Exception e) {
            return false;
        }
    }

    // 从html中获取Icon地址
    private static String getIconUrlByRegex(String urlString) {

        try {
            URL url = new URL(urlString);
            String headString = getHead(urlString);

            if (headString == null || headString.length() == 0) {
                return null;
            }

            for (Pattern iconPattern : ICON_PATTERNS) {
                Matcher matcher = iconPattern.matcher(headString);

                if (matcher.find()) {
                    String iconUrl = matcher.group(1);
                    if (iconUrl.startsWith("http")) {
                        return iconUrl;
                    } else if (iconUrl.startsWith("//")) {
                        return url.getProtocol() + ":" + iconUrl;
                    } else if (iconUrl.startsWith("data:image")) {
                        return iconUrl;
                    } else if (iconUrl.charAt(0) == '/') {// 判断是否为相对路径或根路径
                        iconUrl = url.getProtocol() + "://" + url.getHost() + iconUrl;
                    } else if (iconUrl.startsWith("./")) {// 相对路径的图片
                        iconUrl = urlString + iconUrl.substring(1);
                    } else {
                        iconUrl = urlString + "/" + iconUrl;
                    }
                    return iconUrl;
                }
            }
        } catch (Exception e) {
            logger.error(e, e);
        }
        return null;
    }

    // 爬取一级域名
    private static Set<String> getUrls(String urlString) throws IOException {

        Set<String> urlSet = new HashSet<String>();
        Pattern pattern = Pattern.compile("(http|https)://www\\..+?\\.(aero|arpa|biz|com|coop|edu|gov|info|int|jobs|mil|museum|name|nato|net|org|pro|travel|[a-z]{2})");
        Matcher matcher = pattern.matcher(getHtml(urlString));

        while (matcher.find()) {
            urlSet.add(matcher.group());
        }

        return urlSet;
    }

    // 获取截止到head尾标签的文本
    private static final String getHead2(String urlString) throws IOException {
        HttpURLConnection connection = null;
        BufferedReader reader = null;

        try {
            connection = getConnection(urlString);
            connection.connect();
            reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));

            String line = null;
            StringBuilder headBuilder = new StringBuilder();
            while ((line = reader.readLine()) != null) {
                Matcher matcher = HEAD_END_PATTERN.matcher(line);
                if (matcher.find()) break;
                headBuilder.append(line);
            }

            return headBuilder.toString();
        } finally {
            if (reader != null) reader.close();
            if (connection != null) connection.disconnect();
        }
    }

    private static final String getHead(String urlString) throws IOException {
        HttpResult result = HttpClientUtil.doGet(urlString, null, HttpConnection.getDefaultHeader());
        if (result != null && result.getCode() == 200 && result.getContent() != null) {
            return StringUtil.subString(result.getContent(), "<head>", "</head>");
        }
        return null;
    }

    // 获取html页面文本
    private static final String getHtml(String urlString) throws IOException {
        HttpURLConnection connection = null;
        BufferedReader reader = null;

        try {
            connection = getConnection(urlString);
            connection.connect();
            reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));

            String line = null;
            StringBuilder htmlBuilder = new StringBuilder();
            while ((line = reader.readLine()) != null) {
                htmlBuilder.append(line);
            }

            return htmlBuilder.toString();
        } finally {
            if (reader != null) reader.close();
            if (connection != null) connection.disconnect();
        }
    }

    // 获取一个连接
    private static HttpURLConnection getConnection(String urlString) throws IOException {
        URL url = new URL(urlString);
        // HttpURLConnection.setFollowRedirects(false);
        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
        connection.setInstanceFollowRedirects(false);
        connection.setConnectTimeout(3000);
        connection.setReadTimeout(3000);
        connection.setInstanceFollowRedirects(true);
        connection.setRequestProperty("User-Agent",
                                      "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36");
        return connection;
    }

    private static String getRealUrl(String urlString) throws IOException {
        HttpURLConnection connection = getConnection(urlString);
        int code = connection.getResponseCode();
        if (code == 301 || code == 302) {
            urlString = connection.getHeaderField("Location");
        }
        return urlString;
    }
}
